import glob
import jieba
import sys


def remove_stopwords(words, sw_dict):
    return [w for w in words if not sw_dict.get(w)]


# the data must be orgnised as
# label text
# split by \t
def process_data(raw):
    X = []
    label = []
    for r in raw:
        d = r.split('\t')
        label.append(d[0])
        X.append(jieba.lcut(d[1]))

    return label, X


if __name__ == '__main__':
    input_file = sys.argv[1]

    # 去除停用词
    chinese_stopword_file = sys.argv[2]
    # english_stopword_file = sys.argv[3]

    # 自定义分词词典
    # user_dict = sys.argv[4]
    # jieba.load_userdict(user_dict)
    output_file = sys.argv[3]

    with open(chinese_stopword_file) as fd:
        stopwords = [w.lower().strip() for w in fd.readlines() if w.strip() != '']
    stopwords_dict = dict(zip(stopwords, [1] * len(stopwords)))
    stopwords_dict['\t'] = 1
    stopwords_dict[' '] = 1
    #
    # with open(english_stopword_file) as fd:
    #     stopwords = [w.lower().strip() for w in fd.readlines() if w.strip() != '']
    # english_stopwords_dict = dict(zip(stopwords, [1] * len(stopwords)))
    #
    # for k,v in english_stopwords_dict.items():
    #     stopwords_dict[k] = v

    data = []
    target = []

    out_fd = open(output_file, 'w+')
    with open(input_file) as fd:
        for l in fd.readlines():
            d = l.strip().lower().split('\t')
            if len(d) < 2:
                continue

            # 分词， 去除数字， 去除长度为0， 和长度大于8的词， 有效降低维数
            words = ' '.join([k for k in remove_stopwords(jieba.lcut(d[1]), stopwords_dict) if
                              (not (k.strip().isdigit())) and 0 < len(k.strip()) <= 8])
            words2 = ''
            if len(d) == 3:
                words2 = ' '.join([k for k in remove_stopwords(jieba.lcut(d[2]), stopwords_dict) if
                                  (not (k.strip().isdigit())) and 0 < len(k.strip()) <= 8])

                out_fd.write(d[0] + '\t' + words + '\t' + words2 + '\n')
            else:

                out_fd.write(d[0] + '\t' + words + '\n')







